{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "import math\n",
    "import os\n",
    "import pickle\n",
    "import random\n",
    "import signal\n",
    "from collections import defaultdict\n",
    "from random import shuffle\n",
    "\n",
    "import multitasking\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from itertools import permutations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a b\n",
      "a c\n",
      "b a\n",
      "b c\n",
      "c a\n",
      "c b\n"
     ]
    }
   ],
   "source": [
    "recall_methods = ['a','b','c']\n",
    "for recall_method1, recall_method2 in permutations(recall_methods, 2):\n",
    "    print(recall_method1, recall_method2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<function _signal.default_int_handler>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_threads = multitasking.config['CPU_CORES']\n",
    "multitasking.set_max_threads(max_threads)\n",
    "multitasking.set_engine('process')\n",
    "signal.signal(signal.SIGINT, multitasking.killall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_click = pd.read_pickle('./user_data/data/offline/click.pkl')\n",
    "df_query = pd.read_pickle('./user_data/data/offline/query.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs('./user_data/sim/offline', exist_ok=True)\n",
    "sim_pkl_file = './user_data/sim/offline/itemcf_sim.pkl'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "item_cnt = defaultdict(int)\n",
    "sim_dict = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cal_sim(df):\n",
    "    user_item_ = df.groupby('user_id')['click_article_id'].agg(\n",
    "        lambda x: list(x)).reset_index()\n",
    "    user_item_dict = dict(\n",
    "        zip(user_item_['user_id'], user_item_['click_article_id']))\n",
    "\n",
    "    item_cnt = defaultdict(int)\n",
    "    sim_dict = {}\n",
    "\n",
    "    for _, items in tqdm(user_item_dict.items()):\n",
    "        for loc1, item in enumerate(items):\n",
    "            item_cnt[item] += 1\n",
    "            sim_dict.setdefault(item, {})\n",
    "\n",
    "            for loc2, relate_item in enumerate(items):\n",
    "                if item == relate_item:\n",
    "                    continue\n",
    "\n",
    "                sim_dict[item].setdefault(relate_item, 0)\n",
    "\n",
    "                # 位置信息权重\n",
    "                # 考虑文章的正向顺序点击和反向顺序点击\n",
    "                loc_alpha = 1.0 if loc2 > loc1 else 0.7\n",
    "                loc_weight = loc_alpha * (0.9**(np.abs(loc2 - loc1) - 1))\n",
    "\n",
    "                sim_dict[item][relate_item] += loc_weight  / \\\n",
    "                    math.log(1 + len(items))\n",
    "\n",
    "    for item, relate_items in tqdm(sim_dict.items()):\n",
    "        for relate_item, cij in relate_items.items():\n",
    "            sim_dict[item][relate_item] = cij / \\\n",
    "                math.sqrt(item_cnt[item] * item_cnt[relate_item])\n",
    "\n",
    "    return sim_dict, user_item_dict\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def recall(df_query, item_sim, user_item_dict, worker_id):\n",
    "    data_list = []\n",
    "\n",
    "    for user_id, item_id in tqdm(df_query.values):\n",
    "        rank = {}\n",
    "\n",
    "        if user_id not in user_item_dict:\n",
    "            continue\n",
    "\n",
    "        interacted_items = user_item_dict[user_id]\n",
    "        interacted_items = interacted_items[::-1][:2]\n",
    "\n",
    "        for loc, item in enumerate(interacted_items):\n",
    "            for relate_item, wij in sorted(item_sim[item].items(),\n",
    "                                           key=lambda d: d[1],\n",
    "                                           reverse=True)[0:200]:\n",
    "                if relate_item not in interacted_items:\n",
    "                    rank.setdefault(relate_item, 0)\n",
    "                    rank[relate_item] += wij * (0.7**loc)\n",
    "\n",
    "        sim_items = sorted(rank.items(), key=lambda d: d[1],\n",
    "                           reverse=True)[:100]\n",
    "        item_ids = [item[0] for item in sim_items]\n",
    "        item_sim_scores = [item[1] for item in sim_items]\n",
    "\n",
    "        df_temp = pd.DataFrame()\n",
    "        df_temp['article_id'] = item_ids\n",
    "        df_temp['sim_score'] = item_sim_scores\n",
    "        df_temp['user_id'] = user_id\n",
    "\n",
    "        if item_id == -1:\n",
    "            df_temp['label'] = np.nan\n",
    "        else:\n",
    "            df_temp['label'] = 0\n",
    "            df_temp.loc[df_temp['article_id'] == item_id, 'label'] = 1\n",
    "\n",
    "        df_temp = df_temp[['user_id', 'article_id', 'sim_score', 'label']]\n",
    "        df_temp['user_id'] = df_temp['user_id'].astype('int')\n",
    "        df_temp['article_id'] = df_temp['article_id'].astype('int')\n",
    "\n",
    "        data_list.append(df_temp)\n",
    "\n",
    "    df_data = pd.concat(data_list, sort=False)\n",
    "\n",
    "    os.makedirs('../user_data/tmp/itemcf', exist_ok=True)\n",
    "    df_data.to_pickle(f'../user_data/tmp/itemcf/{worker_id}.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████| 250000/250000 [02:32<00:00, 1639.51it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████| 34739/34739 [00:05<00:00, 6065.10it/s]\n"
     ]
    }
   ],
   "source": [
    "item_sim, user_item_dict = cal_sim(df_click)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "interacted_items = user_item_dict[15]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[277107, 342473, 206415]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "interacted_items"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[206415, 342473]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "interacted_items[::-1][:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "interacted_items = interacted_items[::-1][:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open(sim_pkl_file, 'wb')\n",
    "pickle.dump(item_sim, f)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 召回\n",
    "n_split = max_threads\n",
    "all_users = df_query['user_id'].unique()\n",
    "shuffle(all_users)\n",
    "total = len(all_users)\n",
    "n_len = total // n_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "for path, _, file_list in os.walk('./user_data/tmp/itemcf'):\n",
    "    for file_name in file_list:\n",
    "        os.remove(os.path.join(path, file_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>click_article_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>16</td>\n",
       "      <td>211442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>21</td>\n",
       "      <td>70758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>23</td>\n",
       "      <td>309311</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>68</td>\n",
       "      <td>50644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>75</td>\n",
       "      <td>235323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90464</th>\n",
       "      <td>200004</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90465</th>\n",
       "      <td>200003</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90466</th>\n",
       "      <td>200002</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90467</th>\n",
       "      <td>200001</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90468</th>\n",
       "      <td>200000</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>90469 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       user_id  click_article_id\n",
       "0           16            211442\n",
       "1           21             70758\n",
       "2           23            309311\n",
       "3           68             50644\n",
       "4           75            235323\n",
       "...        ...               ...\n",
       "90464   200004                -1\n",
       "90465   200003                -1\n",
       "90466   200002                -1\n",
       "90467   200001                -1\n",
       "90468   200000                -1\n",
       "\n",
       "[90469 rows x 2 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "90469"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11308"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████| 11308/11308 [02:00<00:00, 93.89it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████| 11308/11308 [01:59<00:00, 94.95it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████| 11308/11308 [02:03<00:00, 91.28it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████| 11308/11308 [01:59<00:00, 94.45it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████| 11308/11308 [02:00<00:00, 93.96it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████| 11308/11308 [02:00<00:00, 93.60it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████| 11308/11308 [02:03<00:00, 91.27it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████| 11308/11308 [02:06<00:00, 89.26it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████| 5/5 [00:00<00:00, 84.97it/s]\n"
     ]
    }
   ],
   "source": [
    "for i in range(0, total, n_len):\n",
    "    part_users = all_users[i:i + n_len]\n",
    "    df_temp = df_query[df_query['user_id'].isin(part_users)]\n",
    "    recall(df_temp, item_sim, user_item_dict, i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "multitasking.wait_for_tasks()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_data = pd.DataFrame()\n",
    "for path, _, file_list in os.walk('../user_data/tmp/itemcf'):\n",
    "    for file_name in file_list:\n",
    "        df_temp = pd.read_pickle(os.path.join(path, file_name))\n",
    "        df_data = df_data.append(df_temp)\n",
    "\n",
    "# 必须加，对其进行排序\n",
    "df_data = df_data.sort_values(['user_id', 'sim_score'],ascending=[True,False]).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_data.to_pickle('./user_data/data/offline/recall_itemcf.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_data = pd.read_pickle('./user_data/data/offline/recall_itemcf.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>article_id</th>\n",
       "      <th>sim_score</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>16</td>\n",
       "      <td>36162</td>\n",
       "      <td>0.100310</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>16</td>\n",
       "      <td>211442</td>\n",
       "      <td>0.059384</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>16</td>\n",
       "      <td>209122</td>\n",
       "      <td>0.052673</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>16</td>\n",
       "      <td>156279</td>\n",
       "      <td>0.041175</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>16</td>\n",
       "      <td>70986</td>\n",
       "      <td>0.037477</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>16</td>\n",
       "      <td>50494</td>\n",
       "      <td>0.006128</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>16</td>\n",
       "      <td>48401</td>\n",
       "      <td>0.006107</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>16</td>\n",
       "      <td>105486</td>\n",
       "      <td>0.006089</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>16</td>\n",
       "      <td>137228</td>\n",
       "      <td>0.006073</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>16</td>\n",
       "      <td>234482</td>\n",
       "      <td>0.006071</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    user_id  article_id  sim_score  label\n",
       "0        16       36162   0.100310    0.0\n",
       "1        16      211442   0.059384    1.0\n",
       "2        16      209122   0.052673    0.0\n",
       "3        16      156279   0.041175    0.0\n",
       "4        16       70986   0.037477    0.0\n",
       "..      ...         ...        ...    ...\n",
       "95       16       50494   0.006128    0.0\n",
       "96       16       48401   0.006107    0.0\n",
       "97       16      105486   0.006089    0.0\n",
       "98       16      137228   0.006073    0.0\n",
       "99       16      234482   0.006071    0.0\n",
       "\n",
       "[100 rows x 4 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_data[df_data['user_id']==16]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tf1.14]",
   "language": "python",
   "name": "conda-env-tf1.14-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
